"""
Case Type   : 操作系统类-文件系统故障
Case Name   : 执行select语句时，备机注入故障系统文件句柄耗尽
Create At   : 2021/7/19
Owner       : @peilinqian
Description :
    step1:建表并插入数据;
    step2:连接备机执行select;
    step3:步骤2执行过程中，备节点注入文件句柄耗尽故障;
    step4:步骤2执行结果验证;
    step5:故障结束后验证集群状态;
    step6:恢复故障节点;
    step7:故障结束后备机再次执行步骤2的查询;
Expect      :
    step1:建表并插入数据;expect:成功
    step2:连接备机执行select;expect:开始执行
    step3:步骤2执行过程中，备节点注入文件句柄耗尽故障;expect:注入成功
    step4:步骤2执行结果验证; expect:执行报错"Too many open files"，
        或者库异常，pg日志报错"Too many open files"
    step5:故障结束后验证集群状态;expect:状态异常
    step6:恢复故障节点;expect:恢复成功
    step7:故障结束后备机再次执行步骤2的查询;expect:成功
History     :
    Modified by @peilinqian 2021/8/23: 优化故障清理
    Modified by @peilinqian 2021/12/9: 优化故障清理后重启数据库
    Modified by @peilinqian 2024/2/7： 优化用例逻辑，精简不必要步骤
    Modified by @peilinqian 2024/7/9： 优化用例，增加清理等待时间
"""
import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ReliabilityFileSystem(unittest.TestCase):
    nodes = ("PrimaryDbUser", "Standby1DbUser", "Standby2DbUser")

    @RestartDbCluster(*nodes)
    def setUp(self):
        self.log = Logger()
        self.pri_sh = CommonSH(self.nodes[0])
        self.sta1_sh = CommonSH(self.nodes[1])
        self.constant = Constant()
        self.com = Common()
        self.sta1_root = Node(node="Standby1Root")
        self.log.info(f'-----{os.path.basename(__file__)} start-----')
        self.inject_time = 60
        self.inject_cmd = f"rfile_deplenish (timeout) values " \
            f"({self.inject_time})"
        self.t_name = "t_os_file_system_case001"
        self.select_sql = f"select pg_sleep(10);" \
            f"select count(*) from pg_settings;" \
            f"select count(*) from {self.t_name};" \
            f"select pg_sleep(20);"
        self.dir_new = os.path.join(macro.PG_LOG_PATH, 'pg_bak',
                                    'pg_log_os_file_system_case001')
        self.init_dir = self.sta1_sh.show_param("log_directory")

    def test_main(self):
        step_txt = '----step0:修改参数log_directory; expect:修改成功----'
        self.log.info(step_txt)
        msg = self.pri_sh.executeGsguc('reload',
                                       self.constant.GSGUC_SUCCESS_MSG,
                                       f"log_directory='{self.dir_new}'",
                                       single=True)
        self.assertTrue(msg, '执行失败:' + step_txt)

        step_txt = '----step1:建表并插入数据; expect:成功----'
        self.log.info(step_txt)
        create_sql = f"drop table if exists {self.t_name} cascade;" \
            f"create table {self.t_name}(id int);" \
            f"insert into {self.t_name} values(generate_series(1,10000));"
        result = self.pri_sh.executDbSql(create_sql)
        self.log.info(result)
        self.assertNotIn("ERROR", result)
        self.assertIn(self.constant.insert_success_msg, result,
                      "执行失败" + step_txt)

        step_txt = '----step2:连接备机执行select; expect:开始执行----'
        self.log.info(step_txt)
        session1 = ComThread(self.sta1_sh.executDbSql,
                             args=(self.select_sql,))
        session1.setDaemon(True)
        session1.start()
        time.sleep(5)

        try:
            step_txt = '----step3:步骤2执行过程中，备节点注入文件句柄耗尽故障; expect:注入成功----'
            self.log.info(step_txt)
            session2 = ComThread(self.com.cfe_inject,
                                 args=(self.sta1_root, self.inject_cmd))
            session2.setDaemon(True)
            session2.start()
            self.start_time = time.time()
            session2.join(60)
            result = session2.get_result()
            self.log.info(result)
            self.assertIn(self.constant.SUCCESS_CFE_FULL_FILENUM_MSG, result,
                          "执行失败" + step_txt)

            step_txt = '----step4:步骤2执行结果验证; ' \
                f'expect:执行报错"{self.constant.TOO_MANYY_FILE_MSG}",' \
                f'或者库异常，pg日志报错"{self.constant.TOO_MANYY_FILE_MSG}"----'
            self.log.info(step_txt)
            session1.join(60)
            result = session1.get_result()
            self.log.info(result)
            if self.constant.FAILED_CONNECT_DB in result or \
                    self.constant.CONNECT_LOST_MSG in result:
                shell_cmd = f'ls -t {self.dir_new} | head -1'
                log_file = os.path.join(self.dir_new,
                                        self.sta1_root.sh(shell_cmd).result())
                self.log.info(log_file)
                shell_cmd = f"cat {log_file} | grep " \
                    f"{self.constant.TOO_MANYY_FILE_MSG}"
                self.log.info(shell_cmd)
                shell_result = self.sta1_root.sh(shell_cmd).result()
                self.log.info(shell_result)
                self.assertNotEqual('', shell_result, '执行失败:' + step_txt)
            else:
                self.assertIn(self.constant.TOO_MANYY_FILE_MSG, result,
                              "执行失败" + step_txt)

        finally:
            text = "-----确认故障清理干净-----"
            self.log.info(text)
            while True:
                time.sleep(5)
                check_cmd = f"df -h"
                check_msg = self.sta1_root.sh(check_cmd).result()
                self.log.info(check_msg)
                if self.constant.TOO_MANYY_FILE_MSG not in check_msg:
                    break
                end_time = time.time()
                if end_time - self.start_time > self.inject_time * 10:
                    break
            step_txt = '----step5:故障结束后验证集群状态; expect:状态异常----'
            self.log.info(step_txt)
            db_status = self.pri_sh.getDbClusterStatus("status")
            self.log.info(db_status)
            self.assertFalse(db_status, "执行失败" + step_txt)

            step_txt = '----step6:恢复故障节点; expect:恢复成功"----'
            self.log.info(step_txt)
            start_result = self.sta1_sh.startDbInstance('standby')
            db_status = self.pri_sh.getDbClusterStatus("status")
            self.log.info(db_status)
            self.assertTrue(start_result, "执行失败" + step_txt)
            self.assertTrue(db_status, "执行失败" + step_txt)

            step_txt = '----step7:故障结束后备机再次执行步骤2的查询; expect:成功----'
            self.log.info(step_txt)
            result = self.sta1_sh.executDbSql(self.select_sql)
            self.log.info(result)
            self.assertIn("10000", result, "执行失败" + step_txt)

    def tearDown(self):
        self.log.info("----this is tearDown----")
        step_txt = '----step8:环境恢复; expect:成功----'
        self.log.info(step_txt)
        drop_sql = f"drop table if exists {self.t_name} cascade;"
        drop_result = self.pri_sh.executDbSql(drop_sql)
        self.log.info(drop_result)
        guc_msg = self.pri_sh.executeGsguc('reload',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           f"log_directory='{self.init_dir}'",
                                           single=True)

        self.log.info(f'-----{os.path.basename(__file__)} end-----')
        self.assertIn(self.constant.TABLE_DROP_SUCCESS, drop_result,
                      "执行失败" + step_txt)
        self.assertTrue(guc_msg, '执行失败:' + step_txt)
